# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy_redis.spiders import RedisCrawlSpider

class BaiduSpider(RedisCrawlSpider):   #继承RedisCrawlSpider 类
    name = 'baidu'
    allowed_domains = ['baidu.com']
    # start_urls = ['http://www.baidu.com/']
    redis_key = 'start_url'   #设置redis键名启动

    rules = (
        # Rule(LinkExtractor(allow=r''), callback='parse_item', follow=True),
        Rule(LinkExtractor(),callback='parse_item',follow=True),
    )

    def parse_item(self, response):
        item = {
            "urls":response.url
        }

        return item
